from scrapy.spiders import Spider
import scrapy
from scrapy_splash import SplashRequest
from scrapy.selector import Selector
from reedlive_scrapy.items import WeChatItem
from shortid import ShortId
from reedlive_scrapy.utils.dbmanager import DbManager
import time
from dateutil import parser
import re


class WeChat(Spider):
    name = 'wechat'

    def start_requests(self):
        self.dbt = DbManager()
        channelList = self.dbt.getItems()
        for item in channelList:
            if item['rsstype'] == 3:
                yield SplashRequest(item['link'],
                                    # yield SplashRequest('http://chuansong.me/n/1499449941121',
                                    args={'wait': 0.5},
                                    callback=self.getWeChatItems,
                                    errback=self.parse_error,
                                    headers={'X-Crawlera-Cookies': 'disable'},
                                    meta={"cookiejar": 1}
                                    )

    def getWeChatItems(self, response):
        # print response.body
        channel = self.dbt.getChannelFromUrl(response.url)
        itemSet = self.parse_wechat(response, channel['_id'])
        # itemSet=self.parse_wechat(response,"aa")
        # itemSet=self.parse_wechat(response,'ddd')

        itemCount = 0
        syncSet = set()
        for item in itemSet:
            if not self.dbt.isWeChatIdExist(item['wid']):
                itemCount = itemCount + 1
                syncSet.add(item)
                yield item

        if itemCount != 0:
            channel['updateTime'] = long(time.time())
            self.dbt.updateChannelTime(channel)
            base_url = 'http://www.iwgc.cn'

            for item in syncSet:
                time.sleep(5)
                itemurl = base_url + '/link/' + item['wid']
                # print itemurl
                yield SplashRequest(itemurl,
                                    meta={
                                        'dont_redirect': True,
                                        'handle_httpstatus_list': [301, 302],
                                        "cookiejar": 1,
                                        'wid': item['_id']
                                    },
                                    dont_filter=True,
                                    callback=self.getContentUrl,
                                    errback=self.parse_error,
                                    args={'wait': 0.5},
                                    )

    def parse_wechat(self, response, channelid):
        selector = Selector(text=response.body)
        rootnode = selector.xpath("//div/div/div/div/div/div[@class='list-group art-list']")
        nodes = rootnode.xpath("a")
        # print nodes
        itemsSet = set()
        for node in nodes:
            item = WeChatItem()
            hre = node.xpath("div")
            mTitle = hre.xpath("h2/text()").extract_first().encode('utf8')
            wid = node.xpath("@href").extract_first()
            if len(wid) > 7:
                wid = wid[6:]
            dat = hre.xpath("span/text()").extract_first()
            if len(dat) > 6:
                dat = dat[3:]
            # print mTitle
            # print wid
            # print dat

            mDate_time = parser.parse(dat)
            mpub = time.mktime(mDate_time.timetuple())
            mTime = time.time()

            item['_id'] = ShortId().generate()
            item['wid'] = wid
            item['title'] = mTitle
            item['published'] = long(mpub)
            item['updated'] = long(mTime)
            item['channelId'] = channelid
            itemsSet.add(item)
        return itemsSet

    def getContentUrl(self, response):
        uri = re.search("window.location.href = '(.*)'", response.body, re.M).group(1)
        # print uri
        # print response.request.meta
        yield SplashRequest(uri,
                            meta={
                                'dont_redirect': True,
                                'handle_httpstatus_list': [301, 302],
                                "cookiejar": 1,
                                'wid': response.request.meta['wid']
                            },
                            headers={'wid': response.request.meta['wid']},
                            dont_filter=True,
                            callback=self.updateWeChatContent,
                            errback=self.parse_error,
                            args={'wait': 0.5},
                            )

    def updateWeChatContent(self, response):
        print "success"
        selector = Selector(text=response.body)
        # content=selector.xpath("//div[@class='rich_media_content ']/").extract_first()
        content = selector.xpath("//body/div/div/div").extract_first()
        content=re.sub('data-src','src',content)
        dr = re.compile(r'<[^>]+>', re.S)
        mSubTitle = dr.sub('', content).strip()
        if len(mSubTitle) > 60:
            mSubTitle = mSubTitle[0:59]

        textCount = self.GetWordNums(dr.sub('', content).strip())

        self.dbt.updateWeChatItem(response.request.meta['wid'], content, mSubTitle, textCount,response.url)

    def parse_error(self, error):
        print error

    def GetWordNums(self, text):
        num = 0
        for i in text:
            if i not in ' \n!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~"':
                num = num + 1
        return num
